import scrapy
import re

import redis
from scrapy_redis.spiders import RedisSpider
from pymongo import MongoClient

from ..items import *
from ..settings import REDIS_HOST,REDIS_PORT,REDIS_DB,REDIS_PARAMS, mongodb

class JdskuSpider(RedisSpider):
    name = 'jdsku'
    allowed_domains = ['jd.com']
    # start_urls = ['https://item.jd.com/100018781645.html#none']
    redis_key = 'jd:start_urls'

    redis_connection = redis.Redis(
        host=REDIS_HOST,
        port=REDIS_PORT,
        password=REDIS_PARAMS.get('password'),
        db=REDIS_DB,
        charset='UTF-8',
        encoding='UTF-8')

    mongo_connection = MongoClient(
        host=mongodb.get('host'),
        port=mongodb.get('port'),
        username=mongodb.get('username'),
        password=mongodb.get('password'))

    # 解决AttributeError: 'JdskuSpider' object has no attribute 'make_requests_from_url'
    def make_requests_from_url(self,url):
        #dont_filter=True带表不去重, 默认False, 去重
        return scrapy.Request(url,
                              meta={'url':url},     #将url传给parse
                              dont_filter=self.settings.get("DONT_FILTER"),
                              cookies=self.settings.get('cookies'))

    def parse(self, response):

        sku = response.meta['url']
        sku_pattern = re.compile("[0-9]+")
        sku = re.search(sku_pattern, sku).group()

        bitWidth = -1
        model = -1
        memoryType = -1
        memoryCapacity = -1
        title = response.xpath('//*[@id="parameter-brand"]/li/a/text()').extract_first()

        print("=========== title =================")
        print(response.xpath('/html'))
        print("=========== title =================")
        li_list = response.xpath('//*[@id="detail"]/div[2]/div[1]/div[1]/ul[2]/li')
        print("***********************************************")
        for li in li_list:
            value = li.xpath('@title').re(".*")[0]
            key = li.re(">.*<")[0][1:-1].replace("：", ":").replace(": {}".format(value), "").replace(":{}".format(value), "")
            print({key:value})
            if key == '显存位宽':
                bitWidth = value
            elif key =='显卡型号':
                model = value
            elif key =='显存类型':
                memoryType = value
            elif key == '显存容量':
                memoryCapacity = value
            elif key == '商品名称':
                title = value
        print("***********************************************")

        item = JdskuspiderItem(
            sku=sku,
            title=title,
            model=model,
            memoryType=memoryType,
            memoryCapacity=memoryCapacity,
            bitWidth=bitWidth
        )
        print("=========== 现在的sku ==========")
        print(sku)

        if bitWidth==-1 and memoryType==-1 and memoryCapacity==-1:
            print("========== no insert into mongodb  ==============")
            if "window.location.href='https://passport.jd.com" in response.xpath('/html').xpath('string(.)').extract_first():
                print("window.location.href页面")
                print("sku重发redis")
                self.redis_connection.lpush("jd:start_urls", 'https://item.jd.com/{}.html'.format(sku))
            else:
                print("非显卡商品, insert -1")
                mongoDB = self.mongo_connection['admin']
                mongoCollection = mongoDB['jd']
                mongoCollection.find_one_and_update(
                    filter={'sku': sku},
                    update={'$set': {
                        'title': title,
                        'model': model,
                        'memoryType': memoryType,
                        'memoryCapacity': memoryCapacity,
                        'bitWidth': bitWidth
                    }}
                )
        else:
            print("==========  insert into mongodb  ==============")
            mongoDB = self.mongo_connection['admin']
            mongoCollection = mongoDB['jd']
            mongoCollection.find_one_and_update(
                filter={'sku':sku},
                update={'$set':{
                    'title':title,
                    'model':model,
                    'memoryType':memoryType,
                    'memoryCapacity':memoryCapacity,
                    'bitWidth': bitWidth
                }}
            )

        yield item

